const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;

const url = "mongodb://taosha_user:apyl8XWP6LSW4JHs@101.35.6.34:27722/taosha";


async function main() {
  const client = await MongoClient.connect(url, {useNewUrlParser: true, useUnifiedTopology: true});
  const db = client.db('taosha');

  let topics = await db.collection('topics')
    .find(
      {sub_tab: '瞎扯', content: /> ./}, {original_url: 1}
    ).toArray();

  for (let i = 0; i < topics.length; i++) {
    let topic = topics[i];

    const browser = await puppeteer.launch({
      headless: true,
      ignoreHTTPSErrors: true,
      args: ['--no-sandbox']
    });

    const page = (await browser.pages())[0];
    await page.setViewport({width: 1500, height: 0});
    await page.setExtraHTTPHeaders({
      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36',
      'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7'
    });


    await page.goto(topic.original_url, {waitUntil: 'domcontentloaded'});

    await page.waitFor('#root > div > main > div > header > div > img');

    let picture = await page.$eval('#root > div > main > div > header > div > img', el => el.src);

    let xiacheBlogs = await page.$$eval('#root > div > main > div > section > div > div > div.content-inner > div', eles => {
      let tmpBlogs = [];
      eles.forEach(el => {
        let question = el.querySelector('h2').innerText;
        if (question) {
          let h = el.innerHTML;


          // 把html标签替换成markdown标签begin
          h = h.replace(/<sup>.*?<\/sup>/g, '$1');
          h = h.replace(/<pre><code.*?>([\s\S]*?)<\/code><\/pre>/g, '```\r\n$1\r\n```');
          h = h.replace(/<span class="LinkCard-content">.*?<\/span>/g, '');
          h = h.replace(/<span class="invisible">.*?<\/span>/g, '');
          h = h.replace(/<span class="visible">(.*?)<\/span>/g, '$1');
          h = h.replace(/<div class="ZhihuDaily-AuthorLine"><span class="ZhihuDaily-Author">.+?<\/span>/, '@知乎用户');
          h = h.replace(/<span class=".*?"><\/span>/, '');
          h = h.replace(/<a class="view-more" href=".*?">进入「知乎」查看相关讨论<\/a>/, '');
          h = h.replace(/<noscript>(.*?)<\/noscript>/g, '');
          h = h.replace(/<blockquote>([\s\S]*?)<\/blockquote>/g, function (item, a) {
            let result = '';
            a = a.split('<br>');
            for (let i = 0; i < a.length; i++) {
              let b = a[i].split('\n');
              for (let j = 0; j < b.length; j++) {
                if (b[j].trim().length > 0) {
                  result += '> ' + b[j].trim() + '\r\n';
                }
              }
            }
            return result;
          });
          h = h.replace(/<ul>([\s\S]*?)<\/ul>/g, function (item, a) {
            return a.replace(/<li>([\s\S]*?)<\/li>/g, function (sub_item, sub_a) {
              return '- ' + sub_a.trim();
            });
          });
          h = h.replace(/<ol>([\s\S]*?)<\/ol>/g, function (item, a) {
            return a.replace(/<li>([\s\S]*?)<\/li>/g, function (sub_item, sub_a) {
              return '- ' + sub_a.trim();
            });
          });
          h = h.replace(/<h2 class="question-title">(.+)<\/h2>/, '### $1');
          h = h.replace(/> <p.*?>(.+?)<\/p>/g, '> $1');
          h = h.replace(/<p.*?>([\s\S]*?)<\/p>/g, '\r\n$1\r\n');
          h = h.replace(/<span.*?>(.*?)<\/span>/g, '$1');
          h = h.replace(/<figure.*?><img.*?src="(.*?)".*?><\/figure>/g, '\r\n![]($1)\r\n');
          h = h.replace(/<img.*?src="(.*?)".*?>/g, '\r\n![]($1)\r\n');
          h = h.replace(/<a.*?href="(.*?)".*?>(.*?)<\/a>/g, '[$2]($1)');
          h = h.replace(/<b>(.*?)<\/b>/g, '**$1**');
          h = h.replace(/<strong><em>\s*(.*?)\s*<\/em><\/strong>/g, ' ***$1*** ');
          h = h.replace(/<em>(.*?)<\/em>/g, ' *$1* ');
          h = h.replace(/<strong.*?>\s*(.*?)\s*<\/strong>/g, ' **$1** ');
          h = h.replace(/<strong.*?>(.*?)<\/strong>/g, '$1');
          h = h.replace(/<strong.*?>(.*?)<\/strong>/g, '$1');
          h = h.replace(/<br.*?>/g, '\r\n');
          h = h.replace(/<h1>(.*?)<\/h1>/g, '### $1');
          h = h.replace(/<h2>(.*?)<\/h2>/g, '### $1');
          h = h.replace(/<h3>(.*?)<\/h3>/g, '### $1');
          h = h.replace(/<u>(.*?)<\/u>/g, '$1');
          h = h.replace(/<i>\s*(.*?)\s*<\/i>/g, ' ***$1*** ');
          h = h.replace(/<address>(.*?)<\/address>/g, '$1');
          h = h.replace(/___/, '\\___');
          h = h.replace(/---/, '\\---');

// 以下是去掉无用的内容
          h = h.replace(/<hr>/g, '');
          h = h.replace(/<div class="meta">[\s\S]*?<\/div>/gim, '');
          h = h.replace(/<div class="answer".*?>/, '');
          h = h.replace(/<div class="answer".*?>/g, '&nbsp;');
          h = h.replace(/<div class="(.*?)".*?>/g, '');
          h = h.replace(/<div id="(.*?)">.*?/g, '');
          h = h.replace(/<div>/g, '');
          h = h.replace(/<\/div>/g, '');
          h = h.replace(/<\/span>/g, '');
          h = h.replace(/<\/a>/g, '');
          h = h.replace(/<figure><\/figure>/g, '');
          h = h.replace(/<iframe.*?>.*?<\/iframe>/g, '');
          h = h.replace(/<wbr>/g, '');
// 把html标签替换成markdown标签end

          tmpBlogs.push({question, h});
        }
      });
      return tmpBlogs;
    });

    let content = `![](${picture})`;
    for (let xiacheBlog of xiacheBlogs) {
      content += `\r\n${xiacheBlog.h}`;
      content += '\r\n***';
    }

    content = content + '\r\n转载自 [知乎日报](https://daily.zhihu.com/)';

    let result = await db.collection('topics').updateOne({_id: topic._id}, {
      $set: {
        content: content
      }
    });



    console.log('ok...', i, topics.length);
  }


}

main();
